{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58210, std: 0.00380, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58225, std: 0.00359, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58225, std: 0.00343, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58188, std: 0.00357, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58188, std: 0.00389, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58243, std: 0.00334, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       " -0.58187835302913937)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=285,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=4, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 522.83390808,  529.42913971,  529.57910852,  532.9222755 ,\n",
       "         529.2507381 ,  454.665903  ]),\n",
       " 'mean_score_time': array([ 1.07981691,  1.01423841,  0.99071655,  1.09062939,  1.0253562 ,\n",
       "         0.88641601]),\n",
       " 'mean_test_score': array([-0.5821032 , -0.58225418, -0.58224565, -0.58187835, -0.58188334,\n",
       "        -0.58242587]),\n",
       " 'mean_train_score': array([-0.48647987, -0.4879834 , -0.49107258, -0.48922089, -0.49059494,\n",
       "        -0.49306166]),\n",
       " 'param_reg_alpha': masked_array(data = [1.5 1.5 1.5 2 2 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_reg_lambda': masked_array(data = [0.5 1 2 0.5 1 2],\n",
       "              mask = [False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([3, 5, 4, 1, 2, 6]),\n",
       " 'split0_test_score': array([-0.57573862, -0.57620368, -0.57709437, -0.57586506, -0.57581825,\n",
       "        -0.5769859 ]),\n",
       " 'split0_train_score': array([-0.48666482, -0.48798072, -0.49168387, -0.48942668, -0.49082753,\n",
       "        -0.49383783]),\n",
       " 'split1_test_score': array([-0.58066312, -0.58102845, -0.58063353, -0.58032394, -0.57995589,\n",
       "        -0.58068411]),\n",
       " 'split1_train_score': array([-0.48595115, -0.48769823, -0.49065892, -0.48871991, -0.49049596,\n",
       "        -0.49292446]),\n",
       " 'split2_test_score': array([-0.58253594, -0.58233113, -0.58146209, -0.58268278, -0.58170878,\n",
       "        -0.58297309]),\n",
       " 'split2_train_score': array([-0.48693881, -0.48726589, -0.49067203, -0.48976627, -0.48991604,\n",
       "        -0.49320786]),\n",
       " 'split3_test_score': array([-0.58476547, -0.58550415, -0.58553298, -0.58435799, -0.5850569 ,\n",
       "        -0.58504732]),\n",
       " 'split3_train_score': array([-0.48746113, -0.48905853, -0.49201217, -0.48931487, -0.49161816,\n",
       "        -0.49329991]),\n",
       " 'split4_test_score': array([-0.58681429, -0.58620467, -0.58650656, -0.58616329, -0.58687838,\n",
       "        -0.58644015]),\n",
       " 'split4_train_score': array([-0.48538342, -0.48791362, -0.4903359 , -0.4888767 , -0.49011702,\n",
       "        -0.49203825]),\n",
       " 'std_fit_time': array([  5.45121764,   8.59471088,   8.87789951,   2.22068399,\n",
       "          6.36853335,  77.08656846]),\n",
       " 'std_score_time': array([ 0.14547554,  0.08207721,  0.04499348,  0.09853197,  0.09254526,\n",
       "         0.13393184]),\n",
       " 'std_test_score': array([ 0.00379601,  0.00358566,  0.00342662,  0.00357017,  0.00388715,\n",
       "         0.00334399]),\n",
       " 'std_train_score': array([ 0.00073341,  0.00059278,  0.0006528 ,  0.00037897,  0.00060003,\n",
       "         0.00059106])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.581878 using {'reg_alpha': 2, 'reg_lambda': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8FPX9x/HXhyQQDjnkEglynwoiBESwXNUKKKCCCoocVtAqVbRqPdpqqa2KShHFn4LIIYooKvehUBBRrnDIGSBGhAAKBLkhkOzn98cMuIQN2UA2s5t8no/HPrKz852ZzwY27/3O8R1RVYwxxpjcVsjrAowxxuRPFjDGGGNCwgLGGGNMSFjAGGOMCQkLGGOMMSFhAWOMMSYkLGCMMcaEhAWMMcaYkLCAMcYYExLRXhfgpXLlymm1atW8LsMYYyLKypUr96lq+ezaFeiAqVatGgkJCV6XYYwxEUVEfgqmne0iM8YYExIWMMYYY0LCAsYYY0xIFOhjMIGcOnWKlJQUTpw44XUpJoDY2Fji4uKIiYnxuhRjTDYsYDJJSUnhkksuoVq1aoiI1+UYP6pKamoqKSkpVK9e3etyjDHZsF1kmZw4cYKyZctauIQhEaFs2bLWuzQmQljABGDhEr7s38aYyGEBY4wxBYkqfPcmHNsf8k3ZMRhjjCkofD6Y9RdIeB8QaDkwpJuzHkwBt3DhQm655ZaLbnM+iYmJXHfddRQpUoTXXnsty3Z9+/alevXqNG7cmMaNG7NmzZoL3qYxJhNfBkwb6IRLq0Fw3cMh36T1YMKcqqKqFCoUud8FLr30UoYPH86UKVOybfvqq6/SvXv3PKjKmAIk4xR88QCs/wzaPgttnoI8OJ5pAXMe/5y+gY27DuXqOhtcXpLnO1953jbbtm2jY8eOtGvXjiVLljBo0CDeeecd0tLSqFmzJmPGjKFEiRLMmjWLxx9/nHLlytGkSROSk5OZMWNGwHUuX76cQYMGcfz4cYoWLcqYMWOoW7fuWW1eeOEFfvjhB3bu3MmOHTt46qmn6N+/PwBHjhyhe/furF+/nqZNmzJhwgREhMGDBzN9+nSOHz9Oy5Yteffdd885EF+hQgUqVKjAzJkzL+I3Z4y5IOlpMPk+SJwBN/wTrh+UZ5uO3K/F+dzmzZvp3bs3X331FaNHj2bevHmsWrWK+Ph4hg4dyokTJ3jggQeYPXs2ixcvZu/eveddX7169Vi0aBGrV69m8ODBPPvsswHbrV27lpkzZ7JkyRIGDx7Mrl27AFi9ejXDhg1j48aNJCcn8+233wIwcOBAVqxYwfr16zl+/PiZgHvnnXd45513cvy+n3vuORo1asRjjz1GWlpajpc3xvg5dQIm9XLCpcMreRouYD2Y88qupxFKVatWpUWLFsyYMYONGzfSqlUrAE6ePMl1111HYmIiNWrUOHPBYc+ePRk5cmSW6zt48CB9+vRh69atiAinTp0K2K5r164ULVqUokWL0q5dO5YvX07p0qVp3rw5cXFxADRu3Jht27Zx/fXXs2DBAoYMGcKxY8fYv38/V155JZ07d+bBBx/M8Xt+6aWXuOyyyzh58iQDBgzglVde4R//+EeO12OMAU4ehYk94cdFcMswiO+X5yWEtAcjIh1EZLOIJInI0wHm9xWRvSKyxn3c7zdviIhsEJFNIjJcHMVEZKaIJLrzXg6wzu4ioiISH8r3FmrFixcHnGMwN954I2vWrGHNmjVs3LiR0aNHo6o5Wt/f//532rVrx/r165k+fXqWFytm3r11erpIkSJnXouKiiI9PZ0TJ07w0EMPMXnyZNatW0f//v0v6iLISpUqISIUKVKEfv36sXz58gtelzEF2olDMKE7bPsGbv0/T8IFQhgwIhIFjAA6Ag2AniLSIEDTSara2H285y7bEmgFNAKuApoBbdz2r6lqPeAaoJWIdPTb5iXAI8CyEL2tPNeiRQu+/fZbkpKSADh27BhbtmyhXr16JCcns23bNgAmTZp03vUcPHiQypUrAzB27Ngs202dOpUTJ06QmprKwoULadasWZZtT4dJuXLlOHLkCJMnT87BOzvX7t27ASdUp0yZwlVXXXVR6zOmQDr+K3xwK6Qsh26joXFPz0oJZQ+mOZCkqsmqehL4GOga5LIKxAKFgSJADPCLqh5T1QUA7jpXAXF+y/0LGALkm7FEypcvz9ixY+nZsyeNGjWiRYsWJCYmUrRoUd5++206dOjA9ddfT8WKFSlVqlSW63nqqad45plnaNWqFRkZGVm2a968OTfffDMtWrTg73//O5dffnmWbUuXLk3//v1p2LAht95661lh5H8M5ueffyYuLo6hQ4fy4osvEhcXx6FDzskTnTp1OnOc55577qFhw4Y0bNiQffv28be//S1HvytjCryjqTCuM/y8Du4cD1fd7m09p0+Dze0H0B14z2/6XuCtTG36AruBtcBkoIrfvNeAA8BB4N8B1l8aSAZquNPXAJ+5zxcC8dnV2LRpU81s48aN57wWrg4fPqyqqj6fT//0pz/p0KFDL2p9zz//vL766qu5UVpIRdK/kTF55tDPqm9dq/qvCqpbvgrppoAEDSIHQtmDCXSSdeYDB9OBaqraCJgHjAMQkVpAfZzeSWWgvYi0PrNikWhgIjBcVZNFpBDwX+Av2RYlMkBEEkQkIbszr8LdqFGjaNy4MVdeeSUHDx7kgQce8LokY4wXDu6EsZ3gwE9w9ydQ+wavKwJANIcHi4Nesch1wAuqepM7/QyAqr6URfsoYL+qlhKRJ4FYVf2XO+8fwAlVHeJOvw8cUdVH3OlSwA/AEXd1lwH7gS6qmpBVjfHx8ZqQcPbsTZs2Ub9+/Qt8194bM2YMb7zxxlmvtWrVihEjRnhUUe6L9H8jY3LVrz85u8WO7Ydek+GKFiHfpIisVNVsT6QK5WnKK4DaIlId2An0AO72byAilVR1tzvZBdjkPt8O9BeRl3B6Qm2AYe4yLwKlgDNnnKnqQaCc33oXAk+cL1zyq379+tGvnzdnjBhj8ljqD064nDwKfaZC5aZeV3SWkAWMqqaLyEBgLhAFvK+qG0RkMM7+u2nAIyLSBUjH6XH0dRefDLQH1uHsVpujqtNFJA54DkgEVrmn0L6l7tlnxhhTYOxJhPFdwJcOfWfAZQ29rugcIb3QUlVnAbMyvfYPv+fPAM8EWC4DOOeAgqqmEPjYTuZ2bS+gXGOMiQw/r4PxXaFQNPSdBRXqeV1RQDZUjDHGRJKdK2HsLRAdC/1mh224gAWMMcZEju1LYfytEFsK+s2CsjW9rui8LGAKuLy4H8yHH35Io0aNaNSoES1btuT777+/4HUZU2D9uAg+uB1KVHB6LmWqeV1RtmywyzB35oKlCL4fTPXq1fn6668pU6YMs2fPZsCAASxblm9G8zEm9LbOg0n3QJnq0HsqXFLR64qCYgFzPrOfdg6m5abLGkLHc8boPEt+ux9My5Ytzzxv0aIFKSkpF/KbM6ZgSpwJn/aF8nXh3qlQvKzXFQUtcr8W53P59X4wo0ePpmPHjue8bowJYMMX8Elv54tpn+kRFS5gPZjzy6anEUr58X4wCxYsYPTo0SxevPhifz3G5H/ffwxT/gRVrnWGf4kt6XVFOWYBE6Yy3w9m4sSJZ81fvXp1jtZ3+n4wX3zxBdu2baNt27YB213I/WASEhKoUqUKL7zwQpb3g1m7di33338/s2fPpmzZyPoWZkyeSxgDMx6D6q2h50QoXNzrii6I7SILc/nhfjDbt2/n9ttv54MPPqBOnTrnrdOYAm/ZuzBjENS+Ee6eFLHhAhYwYS8/3A9m8ODBpKam8tBDD9G4cWPi4yP6ZqPGhM7iYTD7Kah3C9w1AWKKel3RRQnZaMqRINJHUz5y5AglSpRAVXn44YepXbs2jz322AWv74UXXqBEiRI88cQTuVhl7oukfyNjgqIKXw+Bhf+Bq7rBbe9CVIzXVWUp2NGUrQcTwex+MMbkA6ow/59OuDS+B24fFdbhkhPWg4ngHkwgdj8YYyKIKsx5Gpa9A/H3QafXIQIuqg6H+8FELFU952yqSJHf7wdTkL8QmXzG54OZj8PKMdDiIbjpPxChf3eyEv5RmcdiY2NJTU21P2RhSFVJTU0lNjbW61KMuTi+DJj6sBMu1z+eL8MFrAdzjri4OFJSUrK9Mt54IzY29swFn8ZEpIxT8PkA2PA5tHsOWj+ZL8MFLGDOERMTc+bqeGOMyVXpaTD5PkicATcOhlaPel1RSFnAGGNMXjh1HCb1gqR50PFVuHaA1xWFnAWMMcaE2smjMLEH/PgNdB4OTft4XVGesIAxxphQOnEIPrwDUpY7F1BefZfXFeUZCxhjjAmV47/ChG6w+3vo/j5ceZvXFeUpCxhjjAmFo/vgg1th72a48wOo18nrivKcBYwxxuS2wz/D+K7w6zZnuP1aN3hdkScsYIwxJjcdTIFxXZyQuWcyVP+d1xV5xgLGGGNyy6/bYFxnOH4A7v0CrrjW64o8ZQFjjDG5YV8SjO/inJLceypUbuJ1RZ6zgDHGmIu1Z5NzzMWXAX1nwGUNva4oLIR0sEsR6SAim0UkSUSeDjC/r4jsFZE17uN+v3lDRGSDiGwSkeHiKCYiM0Uk0Z33sl/7B0VknbuexSLSIJTvzRhjANi9FsbeDAj0nWnh4idkASMiUcAIoCPQAOiZxR/9Sara2H285y7bEmgFNAKuApoBbdz2r6lqPeAaoJWIdHRf/0hVG6pqY2AIMDRU780YYwBIWQnjboHootBvFlSo53VFYSWUPZjmQJKqJqvqSeBjoGuQyyoQCxQGigAxwC+qekxVFwC461wFxLnTh/yWL+6uwxhjQuOnJc5usaJlnHApW9PrisJOKAOmMrDDbzrFfS2zbiKyVkQmi0gVAFVdAiwAdruPuaq6yX8hESkNdAbm+732sIj8gNODeSQ334wxxpyR/DVMuB0uqQj9ZkOZql5XFJZCGTCBbnCQuVcxHaimqo2AecA4ABGpBdTH6Z1UBtqLSOszKxaJBiYCw1U1+czKVUeoak3gr8DfAhYlMkBEEkQkwe75YozJsa1fwUd3Qumq0HcWlLzc64rCVigDJgWo4jcdB+zyb6Cqqaqa5k6OApq6z28DlqrqEVU9AswGWvgtOhLYqqrDstj2x8CtgWao6khVjVfV+PLly+foDRljCrhNM2BiTyhXxzmgf0lFrysKa6EMmBVAbRGpLiKFgR7ANP8GIlLJb7ILcHo32HagjYhEi0gMzgH+Te4yLwKlgEGZ1lXbb/JmYGsuvhdjTEG3/jP4pDdUuhr6TIfiZb2uKOyF7DoYVU0XkYHAXCAKeF9VN4jIYCBBVacBj4hIFyAd2A/0dRefDLQH1uHsVpujqtNFJA54DkgEVolzm9G33LPPBorIDcAp4FegYNxwwRgTems+gqkPQ5UWcM8nUOQSryuKCKJacE+2io+P14SEBK/LMMaEs4QxMGMQ1GgLPT6CwsW9rshzIrJSVeOzaxfSCy2NMSaiLf0/J1xq/wF6TrJwySELGGOMCeSboTDnaajfGe76EGJiva4o4thYZMYY408VFr4MX78MV3V3bnMcZX8qL4T91owx5jRVmPc8fPsGNO4FXYZDoSivq4pYFjDGGANOuMx5Gpa9A/F/hE6vQSE7inAxLGCMMcbncw7mrxoHLR6Gm/4NEmgwEpMTFjDGmIItI925xmXtx/C7v0D7v1u45BILGGNMwZVxCj67HzZOgXZ/gzZPel1RvmIBY4wpmNLT4NN+sHkm/OFFaPlnryvKdyxgjDEFz6njMKkXJM1zDuY37+91RfmSBYzJF9IzfMzZ8DP7j54kqpAQXUiIKlTI/SlnfkadmS7k/Iw6e77zOgGXjS5UiKgoZ7qQuD8L2b76iJN2BCb2gG2Locub0KS31xXlWxYwJqKpKnM3/MyQuZtJ3ns0z7cvQqZwChRmWYdaITk9HSjQnNcDBl7UuSEaaNkz01G/heJvNQQRwGe1d18X/+07PyVSDoqfOAgf3gEpCXD7SGh0p9cV5WsWMCZiLfkhlVfmJLJmxwFqVyjBu/c2pWnVMmT4lHSf4nN/Zvh8pPuU9Aw9My/D75Hu8531+pllMhSf+r2ekWn+Wdvw/+k7q33G6XVknL3s6WXSTvlI92Wcs+7M63Cmz601HBQSzgqlQpnDLioHARwwFAtlCtZsAvjMdt3XRYhNP8jvlj1AqYOJfH/tUPZFtyE6cU8WNQfu4WYZwO7PiAnaPGIBYyLOxl2HGDI3kYWb91KpVCxDujXi9iaViY4qeBfFqSo+hXSfD5+PwGGZ4YTe6bDMHLT+y5wbmL6z2vs0cNCeN7gDBHugZY+fyjjzeoaPgEH72/zM7+H8QXsph5hQ+CWKyU76n3qU+QvLA7k/knqUG2bn9jTl/AEcINTOrOc8Pdzf1pN1DzdK+C1o/Za/Oq401cqFdvBOCxgTMXbsP8bQr7YwZc1OSsbG8EzHevRpWY3YmII7lIeIOH9AzgxnUjB/F/5Bmzks9dDPlJ7cjahDv/BLx/E8HdeaJ84Ju0w92Qy/QA0QtFn1ks+an+GGoQYKyd+2l3mdJ0/6ztlm4HX4AtYSrH/fdpUFjDGpR9J4a0ESHy7djgg80Lomf2pTk1LFYrwuzYSJc4PWdWAHTL4VDv8CvT6jUrXrvSkwjwQK2nPC0u3Rli1eJOT1WMCYsHU0LZ3Ri39k5KJkjp1M5874Kjx6Q20qlSrqdWkmEuz/EcZ1gRMHoPcUqNLc64pCLsug9YgFjAk7pzJ8fLx8O2/MT2LfkTRuurIiT95Ul1oV7Da1Jkj7tjrhkn4c+kyDy6/xuqICyQLGhA2fT5m5bjevf7mZbanHaF790jNnhhkTtF82wviuoD7oMwMuu8rrigosCxgTFr5N2sfLsxNZt/Mg9S67hDF9m9G2bnk77dPkzO7vYfytEFUY+s6A8nW9rqhAs4Axnlq/8yCvzEnkm637qFy6KK/fcTW3XlOZKLtC3uRUSgJMuB2KlITeU6FsTa8rKvAsYIwnfko9ymtfbmH697soUyyGv91cn14tqhboU47NRfjpO+cK/eLloM90KH2F1xUZLGBMHtt7OI23/reVD5dtJyaqEAPb1WJAmxqUjLVTjs0FSl4IE3tCqTin51Lycq8rMi4LGJMnjqSlM3JRMu99k0xauo8ezarw6O9rU6FkrNelmUi25UtnVOSytZxTkUtU8Loi48cCxoTUyXQfHy37iTf/l0Tq0ZPc3LASf/lDHWqUL+F1aSbSbZru3M+lYgO4dwoUu9TrikwmFjAmJHw+ZfraXbz25WZ27D/OdTXK8nTHelxdpbTXpZn8YN1k+HwAVG4C90yGovb/KhxZwJhcpaos2rqPV2YnsnH3IRpUKsm4+xrSunY5O+XY5I7VH8LUh6FqS7h7EhSxC3DDVUiHnxWRDiKyWUSSROTpAPP7isheEVnjPu73mzdERDaIyCYRGS6OYiIyU0QS3Xkv+7V/XEQ2ishaEZkvIlVD+d7Mub7fcYC7Ry2jz/vLOZx2ijd6NGbGn6+nTR27nsXkkhWjYepDUKOt03OxcAlr2fZgRKQmkKKqaSLSFmgEjFfVA9ksFwWMAG4EUoAVIjJNVTdmajpJVQdmWrYl0MrdFsBioA2wHHhNVReISGFgvoh0VNXZwGogXlWPicifgCHAXdm9P3Pxkvce4bUvNzNr3c+ULV6YFzo34O5rq1I4uuANn29CaMnbMPcZqNMB7hgHMXaCSLgLZhfZZ0C8iNQCRgPTgI+ATtks1xxIUtVkABH5GOgKZA6YQBSIBQoDAsQAv6jqMWABgKqeFJFVQJw7vcBv+aVAryC2Yy7CnkMnGDZ/K5NW7KBIdCEe/X1t+reuQYkitufV5LJvXof5g6F+F+g2GqILe12RCUIwfwl8qpouIrcBw1T1TRFZHcRylYEdftMpwLUB2nUTkdbAFuAxVd2hqktEZAGwGydg3lLVTf4LiUhpoDPwRoB1/hGYHUSN5gIcOnGKkV8nM3rxj6T7fPS69goGtq9N+UtCP/y3KWBUYcF/YNEQaHgH3PoORNkXmEgRzL/UKRHpCfTB+YMOTo8iO4F2ume+G850YKK7++1BYBzQ3u0t1cftnQBfiUhrVV0EICLRwERg+Oke0pmNivQC4nF2qZ1blMgAYADAFVfY1b45ceJUBhOW/sSIBUn8euwUXa6+nL/8oQ5Vy4b2pkWmgFKFr/4O370J1/SCzsMhTIahN8EJJmD6AQ8C/1bVH0WkOjAhiOVSgCp+03HALv8GqprqNzkKeMV9fhuwVFWPAIjIbKAFsMidPxLYqqrD/NcnIjcAzwFtVDUtUFGqOtJdnvj4+PC4oXmYy/ApU1bvZOhXW9h54Di/q12Ov3aox1WVS3ldmsmvfD6Y81dYPhKa3Q8dX4VCdkwv0mQbMO5B+UcARKQMcImqvnz+pQBYAdR2A2kn0AO427+BiFRS1d3uZBfg9G6w7UB/EXkJpyfUBhjmLvMiUAq4P9O6rgHeBTqo6p4g6jPZUFUWbN7DK7M3s/mXwzSsXIoh3RvRqlY5r0sz+ZkvA2YMglXj4bqB8IcXwc5CjEjBnEW2EOePfzSwBtgrIl+r6uPnW849bjMQmItzo/D3VXWDiAwGElR1GvCIiHQB0oH9QF938clAe2Adzm61Oao6XUTicHooicAq99TXt1T1PeBVoATwqfv6dlXtEvRvwpxl5U+/8srsRJZv20+1ssV46+5r6HRVJQrZKMcmlDLSndOQ106C1k9Cu+csXCKYqJ5/L5GIrFbVa9xrVKqo6vMislZVG513wQgQHx+vCQkJXpcRVpL2HGbInM18ufEXypUowqM31KZHsyrERNnuCRNi6Sfh8/th41Ro/zcnYExYEpGVqhqfXbtgjsFEi0gl4E6c3oPJh3YfPM6wr7by6codFCsczV9urMN911enuJ1ybPLCqRPwaV/YMhv+8G9oOTDbRUz4C+avx2Cc3VzfquoKEakBbA1tWSavHDx2ire/TmLst9tQhb4tq/Nwu5qULWGnHJs8cvIYTLoHfvgfdHoNmvf3uiKTS4I5yP8p8KnfdDLQLZRFmdA7cSqDcd9tY8SCJA6npXNb48o8dmMdqlxazOvSTEGSdgQm9oBti6HLW9DkXq8rMrkomIP8ccCbOEO3KM6wLY+qakqIazMhkJ7h4/NVO/nvvC3sPniCdnXL81SHetSvVNLr0kxBc+IgTOgOO1fC7aOg0R1eV2RyWTC7yMbgDA1z+l+/l/vajaEqyuQ+VeWrjb8wZO5mkvYcoXGV0vz3rsa0qFHW69JMQXRsP3xwG/yyAe4YAw26el2RCYFgAqa8qo7xmx4rIoNCVZDJfct/3M8rcxJZ+dOv1ChfnHd6NeGmKy+zEY6NN47shfFdIXUr3DUB6nbwuiITIsEEzD53+JWJ7nRPIPU87U2Y2PzzYYbMSWR+4h4qlizCS7c35I6mcUTbKcfGK4d2w/gucGCHcy+Xmu29rsiEUDABcx/wFvBfnGMw3+EMH2PC1M4Dxxn65RY+X51CiSLRPNWhLv1aVqdoYRvHyXjowA4Y1xmO7oVen0G1Vl5XZEIsmLPItuNcyX+Gu4tsWOAljFd+PXqStxcmMW7JTwD0/10NHmpbk9LFbGhz47H9yTCuq3Ng/94pUKWZ1xWZPHChV9E9jgVM2Dh2Mp0x327jnYU/cPRkOt2axDHoxjpULl3U69KMgb1bnN1i6SegzzS4vLHXFZk8cqEBY0eHw0B6ho9PElIYNm8Lew6ncUP9ijzVoS51KtptZE2Y+GWDc0AfoO9MqHilt/WYPHWhAWPD3HtIVZmz/mdenbuZ5H1Hia9ahhH3NKFZtUu9Ls2Y3+xaAx/cCtGx0HsalK/jdUUmj2UZMCJymMBBIoDte/HIkh9SeXlOIt/vOEDtCiUY1TueG+pXsFOOTXjZsQImdIPYks5usUtreF2R8UCWAaOqtp8ljGzcdYhX5iTy9Za9VCoVy5DujejWJI4oGz7fhJtt38JHd0Lx8tBnOpSukv0yJl+yoXLD3I79x3j9y81M/X4XJWNjeLZTPXpfV43YGDvl2IShHxbAxJ5OqPSeBiUreV2R8ZAFTJhKPZLGm/9L4sNlPxFVSHiwTU0ebFOTUkVjvC7NmMA2z4FPekPZWtB7KpQo73VFxmMWMGHmaFo6oxf/yMhFyRw7mc5dzarw6O/rcFmpWK9LMyZrG6fB5Pucs8Tu/QKK2QknxgImbJxM9/Hxiu0Mn5/EviNpdLjyMp64qS61KpTwujRjzm/tp/DFA1C5KfSaDLGlvK7IhIlghusPdDbZQSAB+It7fxhzgXw+Zea63bz25WZ+Sj1G8+qXMrJ3U5pcUcbr0ozJ3uoJMHUgVG0Fd38MRezcIPObYHowQ4FdOEP2C9ADuAzYDLwPtA1Vcfnd4q37eHnOJtbvPES9yy5hTN9mtK1b3k45NpFh+SiY9YQzYOVdH0Jhu1mdOVswAdNBVa/1mx4pIktVdbCIPBuqwvKzdSkHeWVOIouT9lG5dFGG3nk1XRtXtlOOTeT47i348jmo0xHuGAsxdozQnCuYgPGJyJ3AZHe6u988u6I/B7btO8prX25mxtrdlCkWw99vaUCvFldQJNpOOTYRZNGr8L8XnZuE3f4eRNtgqiawYALmHuAN4G13egnQS0SKAgNDVVh+svdwGsPnb2Xi8u3ERBXiz+1r0b91DUrG2inHJoKowoJ/OwHT6C7o+jZE2XlCJmvBDNefDHTOYvbi3C0nfzl84hSjFiXz3uIfOZnuo0fzKjzSvjYVStruBBNhVOHLv8GSt6BJb7hlGBSynrc5v2DOIosD3gRa4ewSWww8qqopIa4tYqWlZ/DRsu28+b8k9h89yc2NKvHEH+pSvVxxr0szJud8Ppj9JKx4D5r1h45DoJDdFdVkL5j+7RicM8jucKd7ua/dGKqiIpXPp0z9fievf7mFlF+P07JmWZ7uWI9GcaW9Ls2YC+PLgOmPOKcjt/wz3PgvsLMcTZCCCZjyqjrGb3qse0dL41JVvt6yl1fmbGbT7kNceXlJ/nNbQ35Xu5ydcmwiV0Y6THkQ1n0KrZ+Cds9auJgcCaapaow+AAAWBElEQVSfu09EeolIlPvoBaQGs3IR6SAim0UkSUSeDjC/r4jsFZE17uN+v3lDRGSDiGwSkeHiKCYiM0Uk0Z33sl/71iKySkTSRaR75m2FypodB+g5ail9x6zgaFo6b/RozPSB19O6jl3PYiJY+kmY3M8Jl9//A9o/Z+FiciyYHsx9wFvAf3GOwXwH9MtuIRGJAkbg7EpLAVaIyDRV3Zip6SRVHZhp2ZY4x3wauS8tBtoAy4HXVHWBiBQG5otIR1WdDWwH+gJPBPGeLtoPe4/w2tzNzF7/M2WLF+afXa6kZ/MrKBxt+6ZNhDt1Aj7tA1vmwE0vwXUPeV2RiVDBnEW2Heji/5q7i2xYNos2B5JODyUjIh8DXYHMARNws0AsUBhn9IAY4BdVPQYscOs6KSKrgDh3epu7HV8Q678oY7/9kX/N3ERsdCEG3VCb+39XgxJF7HRNkw+cPAYf3w3JC+Dm16HZ/dkvY0wWLvTr9uNBtKkM7PCbTnFfy6ybiKwVkckiUgVAVZfgBMlu9zFXVTf5LyQipXFOn55/AfVflGuuKMO9Lary9VPtGHRDHQsXkz+kHYYPu0PyQug6wsLFXLQL/csYzM7YQG0yX/k/HZioqmki8iAwDmgvIrWA+ri9E+ArEWmtqosARCQamAgMz+lgmyIyABgAcMUVV+Rk0TOurlKaq6vYmWEmHzl+wAmXnaug23vQMM8OY5p87EJ7MMEMEZMC+N8rNQ5n0MzfVqKaqqpp7uQooKn7/DZgqaoeUdUjwGyghd+iI4GtqprdbrpzC1cdqarxqhpfvrzdEMkYju2H8V1g1xq4c5yFi8k1WQaMiBwWkUMBHoeBy4NY9wqgtohUdw/I9wCmZdqG//1UuwCnd4NtB9qISLSIxOAc4N/kLvMiUAqwU6WNuVhH9sDYW2BPIvT4COpnNWiHMTmX5S4yVb2oGzuoarqIDATmAlHA+6q6QUQGAwmqOg14RES6AOnAfpyzwMAZWLM9sA6ntzRHVae7owo8ByQCq9zTgN9S1fdEpBnwBVAG6Cwi/1TVKy/mPRiTrx3aBeO7woEdcPckqNnO64pMPiOqBXdA5Pj4eE1ISPC6DGPy3oHtMK4zHN0H93wKVVt6XZGJICKyUlXjs2tnpz8ZU9Ck/uD0XNIOQe+pEJft3wljLogFjDEFyd4tTs8l4yT0mQ6Vrva6IpOPWcAYU1D8ssHpuSDQdyZUbOB1RSafs3FNjCkIdq2GsTdDoRjoN8vCxeQJCxhj8rsdy2FcFyh8iRMu5Wp7XZEpICxgjMnPti2G8bdC8XJOuFxa3euKTAFiAWNMfpU0HyZ0h1Jx0HcWlK6S/TLG5CILGGPyo81zYGIPKFvTOaBfslL2yxiTyyxgjMlvNkyBSfdAxSudU5FL2Jh7xhsWMMbkJ2s/ce5EWbmpcxFlsUu9rsgUYBYwxuQXq8bD5wOgaivo9TnElvK6IlPAWcAYkx8sHwXT/gw128Pdn0CREl5XZIwFjDER77s3YdYTULcT9JwIhYt5XZExgA0VY0xk+/pVWPAiNLjVuRNlVIzXFRlzhgWMMZFIFf73L/jmdWjUA7qOgCj7OJvwYv8jjYk0qjD3OVg6Apr0hlvegEK2t9uEHwsYYyKJz+ccb0kYDc0fgA4vW7iYsGUBY0yk8GXAtEdgzQRo9Sjc8E9wbhtuTFiygDEmEmScgi8ehPWToc3T0PZpCxcT9ixgjAl36Sedq/MTZ8Dvn4ffPe51RcYExQLGmHB26gR80hu2zoWbXoLrHvK6ImOCZgFjTLg6eRQ+vhuSF8It/4X4+7yuyJgcsYAxJhylHYYP74QdS+HW/4PGd3tdkTE5ZgFjTLg5fgAmdINdq52r86/q5nVFxlwQCxhjwsnRVPjgVtizCe4cB/U7e12RMRfMAsaYcHFkD4zvCqk/QI+PoM4fvK7ImItiAWNMODi0C8Z1gUM74Z5PoEZbrysy5qKFdIwJEekgIptFJElEng4wv6+I7BWRNe7jfr95Q0Rkg4hsEpHh4igmIjNFJNGd97Jf+yIiMsnd1jIRqRbK92ZMrjmwHcZ0hMM/OzcKq9HW64qMyRUhCxgRiQJGAB2BBkBPEWkQoOkkVW3sPt5zl20JtAIaAVcBzYA2bvvXVLUecA3QSkQ6uq//EfhVVWsB/wVeCdFbMyb3pP4A73eE4786tziuep3XFRmTa0LZg2kOJKlqsqqeBD4Guga5rAKxQGGgCBAD/KKqx1R1AYC7zlVAnLtMV2Cc+3wy8HsRG0vDhLE9iTCmE5w6Bn2mQ1xTrysyJleFMmAqAzv8plPc1zLrJiJrRWSyiFQBUNUlwAJgt/uYq6qb/BcSkdJAZ2B+5u2pajpwECibe2/HmFz08zoYezOoD/rNgkpXe12RMbkulAETqPegmaanA9VUtREwD7cHIiK1gPo4vZPKQHsRaX1mxSLRwERguKom52B7iMgAEUkQkYS9e/fm8C0Zkwt2roKxt0B0Eeg3GyrU97oiY0IilAGTAlTxm44Ddvk3UNVUVU1zJ0cBp/cR3AYsVdUjqnoEmA208Ft0JLBVVYcF2p4bQKWA/ZmLUtWRqhqvqvHly5e/4DdnzAXZvsw5FTm2pNNzKVfL64qMCZlQBswKoLaIVBeRwkAPYJp/AxGp5DfZBTi9G2w70EZEokUkBucA/yZ3mRdxwmNQpu1NA/q4z7sD/1PVc3owxnjmx2/gg9ugeHmn51KmmtcVGRNSIbsORlXTRWQgMBeIAt5X1Q0iMhhIUNVpwCMi0gVIx+lt9HUXnwy0B9bh7Oaao6rTRSQOeA5IBFa5x/Dfcs8+Gw18ICJJ7rp6hOq9GZNjSfPg43ucUOk9FS65zOuKjAk5Kchf8uPj4zUhIcHrMkx+lzgLPu0D5epC7ylQvJzXFRlzUURkparGZ9fObuZtTCht+AI+uRcqXgV9plm4mALFAsaYUPl+Eky+DyrHO7vFil3qdUXG5CkLGGNCYeU4+OIBqNoKen3mnDVmTAFjAWNMbls2EqY/ArV+D/d8CkVKeF2RMZ6wgDEmN337Bsx+Eure7Ay5H1PU64qM8YwN129MblCFRa/Cgn/DlbfB7aMgKsbrqozxlAWMMRdLFeYPhsVD4eqe0HUEFIryuipjPGcBY8zFUIW5z8LSt6FpX7j5v1DI9jwbAxYwxlw4nw9mPg4rx8C1D0KHl8HuEGHMGRYwxlwIXwZM+zOs+RBaDYIbXrBwMSYTCxhjcirjlHONy/rPoO0z0OavFi7GBGABY0xOpKc5V+cnzoAb/gnXZx7U2xhzmgWMMcE6dRw+6Q1bv4QOr0CLB72uyJiwZgFjTDBOHoWJPZx7utwyDOL7eV2RMWHPAsaY7Jw4BB/dCTuWwa3/B417el2RMRHBAsaY8zn+K0zoBrvWQLf34KpuXldkTMSwgDEmK0dT4YOusHcz3PUB1LvZ64qMiSgWMMYEcvgXGN8Vfv0RekyE2jd4XZExEccCxpjMDu6E8V3g0C64+xOo0cbrioyJSBYwxvj79ScY1xmO7Yd7v4ArWnhdkTERywLGmNNSf3DC5eQR5xbHcU29rsiYiGYBYwzAnkRnt5gvHfrMgEqNvK7ImIhn44obs3stjO3kPO87y8LFmFxiAWMKtp0rYdwtEB0L/WZDhXpeV2RMvmEBYwqu7UthXFeILQ39ZkHZml5XZEy+YgFjCqYfF8EHt0OJCk7PpUw1rysyJt+xgDEFz9Z58OEdUPoKJ1xKVfa6ImPypZAGjIh0EJHNIpIkIk8HmN9XRPaKyBr3cb/fvCEiskFENonIcBHnjk4i8m8R2SEiRzKtq6qIzBeRtSKyUETiQvneTIRKnAkf94RytaHvTLikotcVGZNvhSxgRCQKGAF0BBoAPUWkQYCmk1S1sft4z122JdAKaARcBTQDTl9OPR1oHmA9rwHjVbURMBh4KTffj8kH1n/u3M/lsobQZzoUL+t1Rcbka6HswTQHklQ1WVVPAh8DXYNcVoFYoDBQBIgBfgFQ1aWqujvAMg2A+e7zBTnYlikI1kyEz/4Icc3g3ilQtIzXFRmT74UyYCoDO/ymU9zXMuvm7taaLCJVAFR1CU5I7HYfc1V1Uzbb+x44PZb6bcAlImJfUQ0kjIEpf4Jq10OvzyC2pNcVGVMghDJgJMBrmml6OlDN3a01DxgHICK1gPpAHE4otReR1tls7wmgjYisxtmdthNIP6cokQEikiAiCXv37s3J+zGRaNm7MGMQ1LrBGbiycHGvKzKmwAhlwKQAVfym44Bd/g1UNVVV09zJUcDpwZ9uA5aq6hFVPQLMBs476qCq7lLV21X1GuA597WDAdqNVNV4VY0vX778hbwvEykWD4PZT0G9W6DHhxBT1OuKjClQQhkwK4DaIlJdRAoDPYBp/g1EpJLfZBfg9G6w7Ti9kWgRicHpkZx3F5mIlBOR0+/nGeD9XHgPJhKpwsKXYd7zzh0o7xgL0UW8rsqYAidkAaOq6cBAYC5OOHyiqhtEZLCIdHGbPeKeivw98AjQ1319MvADsA7n2Mr3qjodzpy+nAIUE5EUEXnBXaYtsFlEtgAVgX+H6r2ZMKYK816AhS/B1XfD7aMgKsbrqowpkEQ182GRgiM+Pl4TEhK8LsPkFlWY8zQsewea9oObh0Ihu5bYmNwmIitVNT67djZcv8l9Pp8z7L3vlPMzI/236YxT4Mvwm3ee6Qz3tdOPgNN+bfduhs0z4do/QYeXQAKdZ2KMySsWMHlNFdTn98fy1Nl/gDP/Qc5y+kLaZv5jnlttM81TX97/XgtFQ1RhaP0UtHvWwsWYMGABcyHWTYaVY88OCV/GeUIj0zwvFIpxjkUUiv7t4T+d1bzoIn7TUdmvJ3PbYLeTk5oCTVugGBN2LGAuxOleSHRhKFQ8iz+qp/8Q+k/HnOcPcBbTOWmb5R/oKK9/Y8aYAsgC5kI0usN5GGOMyZKdYmOMMSYkLGCMMcaEhAWMMcaYkLCAMcYYExIWMMYYY0LCAsYYY0xIWMAYY4wJCQsYY4wxIVGgR1MWkb3AT17X4accsM/rIs4j3OuD8K8x3OuD8K8x3OuD/F9jVVXN9o6NBTpgwo2IJAQzBLZXwr0+CP8aw70+CP8aw70+sBpPs11kxhhjQsICxhhjTEhYwISXkV4XkI1wrw/Cv8Zwrw/Cv8Zwrw+sRsCOwRhjjAkR68EYY4wJCQuYPCYiHURks4gkicjTWbS5U0Q2isgGEfko3GoUkStEZIGIrBaRtSLSKY/re19E9ojI+izmi4gMd+tfKyJNwqy+e9y61orIdyJydV7WF0yNfu2aiUiGiHTPq9r8tp1tjSLSVkTWuJ+Vr8OpPhEpJSLTReR7t75+eVxfFfdzusnd/qMB2oT2s6Kq9sijBxAF/ADUAAoD3wMNMrWpDawGyrjTFcKwxpHAn9znDYBteVxja6AJsD6L+Z2A2YAALYBlYVZfS79/3455XV8wNfr9X/gfMAvoHm41AqWBjcAV7nRef1ayq+9Z4BX3eXlgP1A4D+urBDRxn18CbAnwWQ7pZ8V6MHmrOZCkqsmqehL4GOiaqU1/YISq/gqgqnvCsEYFSrrPSwG78rA+VHURzoc1K12B8epYCpQWkUp5U1329anqd6f/fYGlQFyeFHZ2Ddn9DgH+DHwG5PX/QSCoGu8GPlfV7W77PK0ziPoUuEREBCjhtk3Pi9oAVHW3qq5ynx8GNgGVMzUL6WfFAiZvVQZ2+E2ncO4/eB2gjoh8KyJLRaRDnlXnCKbGF4BeIpKC8+32z3lTWtCCeQ/h4o843yDDiohUBm4D3vG6lvOoA5QRkYUislJEentdUCZvAfVxvoCtAx5VVZ8XhYhINeAaYFmmWSH9rETn1opMUCTAa5lP44vG2U3WFueb7TcicpWqHghxbacFU2NPYKyqvi4i1wEfuDV68uEJIJj34DkRaYcTMNd7XUsAw4C/qmqG8wU8LEUDTYHfA0WBJSKyVFW3eFvWGTcBa4D2QE3gKxH5RlUP5WURIlICpyc6KMC2Q/pZsYDJWylAFb/pOM7dvZQCLFXVU8CPIrIZJ3BW5E2JQdX4R6ADgKouEZFYnHGNPNmVEkAw78FTItIIeA/oqKqpXtcTQDzwsRsu5YBOIpKuqlO8LessKcA+VT0KHBWRRcDVOMcawkE/4GV1DnYkiciPQD1geV4VICIxOOHyoap+HqBJSD8rtossb60AaotIdREpDPQApmVqMwVoByAi5XB2AySHWY3bcb41IiL1gVhgbx7WmJ1pQG/3DJkWwEFV3e11UaeJyBXA58C9YfRt+yyqWl1Vq6lqNWAy8FCYhQvAVOB3IhItIsWAa3GOM4QL/89JRaAuefhZdo/9jAY2qerQLJqF9LNiPZg8pKrpIjIQmItzhs77qrpBRAYDCao6zZ33BxHZCGQAT+blN9wga/wLMEpEHsPpTvd1v6XlCRGZiLMLsZx7HOh5IMat/x2c40KdgCTgGM43yTwTRH3/AMoCb7s9hHTN44ERg6jRc9nVqKqbRGQOsBbwAe+p6nlPu87L+oB/AWNFZB3Orqi/qmpejrDcCrgXWCcia9zXngWu8KsxpJ8Vu5LfGGNMSNguMmOMMSFhAWOMMSYkLGCMMcaEhAWMMcaYkLCAMcYYExIWMMYYY0LCAsYYj4hIXxF5K5fWtc29MDe7dkdyY3vGBMMCxpgccq96ts+OMdmwD4kxQRCRau6Nm94GVgH3isgSEVklIp+6AwoiIp1EJFFEFrs3cpoR5Po7i8gycW7iNs8dWgQReUFExonIl24v5XYRGSIi60RkjjvW1GlPishy91HLXb66W+cKEfmX3/ZKiMh8t/51IpL5lgzGXDQLGGOCVxcYD9yIM+DnDaraBEgAHncH/XwXZwDL63FuMhWsxUALVb0G5x48T/nNqwncjHPvjgnAAlVtCBx3Xz/tkKo2xxkmfpj72hvA/6lqM+Bnv7YngNvc+tsBr0sYD5tsIpMFjDHB+8m9KVMLnDt5fuuO8dQHqIozUm6yqv7otp+Yg3XHAXPdcaueBK70mzfbHV17Hc74cHPc19cB1fzaTfT7eZ37vJXf6x/4tRXgPyKyFpiHcw+Qijmo15hs2WCXxgTvqPtTgK9Utaf/TBG55iLW/SYwVFWniUhbnJu6nZYGoKo+ETnlN7Coj7M/wxrE89PuwelhNVXVUyKyDWdUbGNyjfVgjMm5pUArv+McxUSkDpAI1HDvHghwVw7WWQrY6T7vc4F13eX3c4n7/FucWy6AEyr+29vjhks7nB6YMbnKejDG5JCq7hWRvsBEESnivvw3Vd0iIg8Bc0RkHzm7sdQLwKcishMnwKpfQGlFRGQZzhfH072rR4GPRORRnBtPnfYhMF1EEnDuuph4Adsz5rxsuH5jcpGIlFDVI+4B8xHAVlX9r9d1GeMF20VmTO7q7x7434CzG+pdj+sxxjPWgzEmxESkH86uKn/fqurDXtRjTF6xgDHGGBMStovMGGNMSFjAGGOMCQkLGGOMMSFhAWOMMSYkLGCMMcaExP8DPUKrWLbVDy0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bfe63d0c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_lambda' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加正则项后的分数(0.5818)要比不加正则项的分数(0.5819)效果好"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 由于最佳值处于边缘地带，在边缘地带继续探索一下\n",
    "# 进一步增大reg_alpha,进一步减小 lambda尝试一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [2, 2.5, 3], 'reg_lambda': [0.1, 0.3, 0.5]}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha2 = [ 2, 2.5,3]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda2 = [0.1,0.3, 0.5]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test6_1 = dict(reg_alpha=reg_alpha2, reg_lambda=reg_lambda2)\n",
    "param_test6_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58193, std: 0.00351, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.58225, std: 0.00373, params: {'reg_alpha': 2, 'reg_lambda': 0.3},\n",
       "  mean: -0.58188, std: 0.00357, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58236, std: 0.00344, params: {'reg_alpha': 2.5, 'reg_lambda': 0.1},\n",
       "  mean: -0.58205, std: 0.00370, params: {'reg_alpha': 2.5, 'reg_lambda': 0.3},\n",
       "  mean: -0.58230, std: 0.00341, params: {'reg_alpha': 2.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58190, std: 0.00346, params: {'reg_alpha': 3, 'reg_lambda': 0.1},\n",
       "  mean: -0.58194, std: 0.00371, params: {'reg_alpha': 3, 'reg_lambda': 0.3},\n",
       "  mean: -0.58184, std: 0.00349, params: {'reg_alpha': 3, 'reg_lambda': 0.5}],\n",
       " {'reg_alpha': 3, 'reg_lambda': 0.5},\n",
       " -0.58184150548318958)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb6_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=285,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch6_1 = GridSearchCV(xgb6_1, param_grid = param_test6_1, scoring='neg_log_loss',n_jobs=4, cv=kfold)\n",
    "gsearch6_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch6_1.grid_scores_, gsearch6_1.best_params_,     gsearch6_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "得到的分数略好于 alpha=2 lambda=0.5时的分数"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
